# -*- coding: utf-8 -*-
import time

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from DataSpider.items import ShixisengItem, ShixisengItemLoader
from utils.common import get_md5
from datetime import datetime
from scrapy.conf import settings
from selenium import webdriver
from scrapy.xlib.pydispatch import dispatcher
from scrapy import signals
from scrapy.http import HtmlResponse


class ShixisengSpider(CrawlSpider):
    name = 'shixiseng'
    allowed_domains = ['www.jrzp.com']
    start_urls = ['https://www.jrzp.com/']
    rules = (
        # Rule(LinkExtractor(allow=r'job/.*'), follow=True),
        Rule(LinkExtractor(allow=r'job\d+.*'), callback='parse_detail', follow=True),
        # Rule(LinkExtractor(allow=r'interns?.*keyword=.*/'), follow=True),
        # Rule(LinkExtractor(allow=r'intern/.*'), callback='parse_job', follow=True),
    )
    headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Accept-Encoding': 'gzip, deflate, br',
        "Connection": "keep - alive",
        # "Cookie": 'gr_user_id=96f071c5-6f26-472c-aa73-e847c57f2b7f; __jsluid_s=e99611122f1b69b78bf6e183b52c4708; MEIQIA_TRACK_ID=1YfbLQKoTUdzS9tbdAYQPIts202; MEIQIA_VISIT_ID=1YvvoUqXt8m8uUKRzx2bLMexeGf; affefdgx=usr_hyqswn8onb1a; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%22170c4017d4212d-0be2ef0ec558c5-7711a3e-1327104-170c4017d43d9%22%2C%22%24device_id%22%3A%22170c4017d4212d-0be2ef0ec558c5-7711a3e-1327104-170c4017d43d9%22%2C%22props%22%3A%7B%7D%7D; uuid=d9c358d5-e0eb-95be-8151-a5c397d53344; search=; Hm_lvt_03465902f492a43ee3eb3543d81eba55=1583836050,1583934226,1583934284; sxs_usr="2|1:0|10:1583934308|7:sxs_usr|24:dXNyX2h5cXN3bjhvbmIxYQ==|c3f84daa59dc7892acd516d3cd016e067c82b8848f0f59ebcdfe85135e73b39b"; userflag=user; SXS_XSESSION_ID="2|1:0|10:1583934308|15:SXS_XSESSION_ID|48:ZGMwOGViOTAtMDU1My00OWViLWI0OTUtZTIzYjIzMjk1MWZi|3589d4c5a076a529d8db7b65391c9903c6f8c936a4f6a0ade99e7ac8863d6c7e"; SXS_XSESSION_ID_EXP="2|1:0|10:1583934308|19:SXS_XSESSION_ID_EXP|16:MTU4NjUyNjMwOA==|46e895838d1a5cf3a3c03d9ba9fccdd1f9e44ebcaf36c9d2b9820398ecb87d83"; xyz_usr="2|1:0|10:1583934308|7:xyz_usr|24:dXNyX2h5cXN3bjhvbmIxYQ==|b527a146cac5f27143e9e83e1e76b71c308cc0d7109855aa08f47ad86238ac12"; uid1=331d9839-7c6e-bf32-8eb4-f6a125722b7b; uid2=ba5691b8-fadb-6feb-a057-762930eb4655; gr_session_id_96145fbb44e87b47=280c4457-820d-41ee-9def-226cafb19430; gr_cs1_280c4457-820d-41ee-9def-226cafb19430=user_id%3Ausr_hyqswn8onb1a; gr_session_id_96145fbb44e87b47_280c4457-820d-41ee-9def-226cafb19430=true; SXS_VISIT_XSESSION_ID_V3.0="2|1:0|10:1583939335|26:SXS_VISIT_XSESSION_ID_V3.0|48:NTc0NTE3YzAtNzJkZi00Mzk2LWExMmItMjMwMjhlOWNiMzA3|5eff7f5d0c69a92a1c7861dd2c347cd58c48f655f1ce98797f0d8e0c243c2c77"; SXS_VISIT_XSESSION_ID_V3.0_EXP="2|1:0|10:1583939335|30:SXS_VISIT_XSESSION_ID_V3.0_EXP|16:MTU4NjUzMTMzNQ==|4e083388b99619cf8fb832b89a41ed7c419a6398fd027b9d5d24de70fa35f13a"; Hm_lpvt_03465902f492a43ee3eb3543d81eba55=1583939336',
        'cache-control': 'max-age=0',
        'if-none-match': ' W / "a50b034d5e404ed2970a77233185690d218582c3"',
        'sec-fetch-dest': 'document',
        'sec-fetch-mode': 'navigate',
        'sec-fetch-site': 'none',
        'sec-fetch-user': '?1',
        'upgrade-insecure-requests': 1,
    }

    # def __init__(self):
    #     # 设置chrome不加载图片
    #     self.chrome_opt = webdriver.ChromeOptions()
    #     self.prefs = {"profile.managed_default_content_settings.images": 2}
    #     self.chrome_opt.add_experimental_option("prefs", self.prefs)
    #     self.browser = webdriver.Chrome(executable_path="D:/scrapy/webdriver/chromedriver_win32/chromedriver.exe",
    #                                     chrome_options=self.chrome_opt)
    #     super(ShixisengSpider, self).__init__()
    #     # 信号监控，并处理相应函数
    #     dispatcher.connect(self.spider_closed, signals.spider_closed)
    #     dispatcher.connect(self.login_account, signals.spider_opened)
    #
    # def login_account(self):
    #     self.browser.get("https://www.shixiseng.com/")
    #     self.browser.find_element_by_css_selector("#log_name").send_keys("13395993129")
    #     self.browser.find_element_by_css_selector(".password").send_keys("123456")
    #     self.browser.find_element_by_css_selector("#index_login_btn").click()
    #
    # def spider_closed(self, spider):
    #     print("spider closed")
    #     # print(self.browser.page_source)
    #     self.browser.quit()

    # def parse_start_url(self, response):
    #     return []

    def parse_detail(self, response):
        # 保存当前网页
        # with open("D:\check.html", "wb") as f:
        #     f.write(response.body)

        item_loader = ShixisengItemLoader(item=ShixisengItem(), response=response)
        item_loader.add_css("title", ".left .zwm span ::text")
        item_loader.add_value("url", response.url)
        item_loader.add_value("url_object_id", get_md5(response.url))
        item_loader.add_css("salary_min", ".left .zwm em ::text")
        item_loader.add_css("salary_max", ".left .zwm em ::text")
        item_loader.add_css("job_city", ".gzddCon span::text")
        item_loader.add_value("work_week_time", "5")
        item_loader.add_value("degree_need",  response.css(".zwjbnr .left .jbyq span:nth-child(3)::text").extract()[1])
        item_loader.add_value("work_how_long", "5")
        item_loader.add_value("publish_time",  response.css(".zwjbnr .left .gsmc span:nth-child(2)::text").extract()[1])
        item_loader.add_value("job_advantage", "木有")
        item_loader.add_css("job_desc", ".zwmsCon ::text")
        item_loader.add_css("job_addr", ".gzddCon span::text")
        company_url = response.css(".gsxq ul li a ::attr(href)").extract_first()
        if company_url is None:  company_url = "暂无网址"
        item_loader.add_value("company_url", company_url)
        item_loader.add_css("company_name", ".gsxq .gsmc::text")
        item_loader.add_value("crawl_time", datetime.now())

        job_item = item_loader.load_item()
        return job_item
